var superagent=require('superagent')
var cheerio=require('cheerio')
var async=require('async')
var fs = require('fs');
var xlsx = require('better-xlsx');





function saveExcel(data,typeName,topName){
    var file = new xlsx.File();
    var sheet = file.addSheet('Sheet1');
    for(var i=0;i<data.length;i++){
        var row = sheet.addRow();
        var cell1 = row.addCell();
        var cell2 = row.addCell();
        var cell3 = row.addCell();
        var cell4 = row.addCell();
        var cell5 = row.addCell();
        var cell6 = row.addCell();
        cell1.value = data[i]['skus'][0]['name'];
        cell2.value = data[i]['skus'][0]['desc'];
        cell3.value = typeName;
        cell4.value = data[i]['skus'][0]['std_unit_name'];
        cell5.value = data[i]['skus'][0]['std_sale_price']/100;
        cell6.value = data[i]['skus'][0]['img_url'];
    }
    
    file
      .saveAs()
      .pipe(fs.createWriteStream(topName + '/'+typeName+'.xlsx'))
      .on('finish', () => console.log('Done.'));
}




var urls=[]
var url='https://bshop.guanmai.cn/v587/product/category/get'

var jihaiUrl='https://www.amazon.com/gp/search?ie=UTF8&qid=1509371076&lo=computers&rh=n%3A172282%2Cn%3A%21493964%2Cn%3A541966%2Cn%3A13896617011%2Cn%3A565098%2Cn%3A13896603011&page=4'
var cookie='skin=noskin; x-wl-uid=1bEhG+6sSOEkK13Pv/RwG2ihh4yusUVWtiR7jdnCVBirBtF1G4fFn5iRTkGx/p4ZnNcO036w242Y=; session-token=2ZO439BK7eO49YOzQsOfI6KzK4XuAu5iM2GCW6mvTL05L+GhReCJNcFkYVOF8qeHWUHHZ+SQBWYJBcTKnB69QLW0430D5Vac5fBcIzWam+89JI7QHIkyil2i8QYkh+rZ49CAzohL0cTcVk2nC113m4mDjS8iGWzrzBjS/1WvFqMriXRebUNQegQhz34kuqF7HiDOA+Xwu0FmB5QIu7eBk2e+f6W29C0hI04OWD6jpe0MWHy0mZUzi5sbruerqKD2; csm-hit=s-AS6FNTQJ86PWPABQXGQC|1509371140395; ubid-main=134-1468957-3633231; session-id-time=2082787201l; session-id=142-7387583-4234639'
superagent.get(jihaiUrl)
    .set("Cookie",cookie)
    .end(function(err,res){
        if(err)
            console.error('错误')
        else{
            console.log(res.text)
            // var erji=JSON.parse(res.text)['data']
            // for(var i=0;i<erji.length;i++){
            //     fs.mkdir(erji[i].name)
            //     for(var j=0;j<erji[i].children.length;j++){
            //         urls.push({url:'https://bshop.guanmai.cn/v587/product/sku/get?level=2&category_id='+erji[i].children[j]['id'],name:erji[i].children[j]['name'],topName:erji[i]['name']})
            //     }
                
            // }
            // console.log(urls)
            // getData()
        }
    })

function getData(){
    var concurrencyCount = 0
    var itemArr = []
    var fetchUrl = function(url, callback){
        // var delay = parseInt((Math.random() * 10000000) % 2000, 10)
        var delay = 1000
        concurrencyCount ++
        console.log('并发数：'+ concurrencyCount,'正在抓取：',url.url,'耗时：（ms）'+delay)
        superagent.get(url.url)
            .set("Cookie",cookie)
            .end(function(err,res){
                var data=JSON.parse(res.text).data
                saveExcel(data,url.name,url.topName)
            })
        setTimeout(function(){
            concurrencyCount --
            callback(null,url.url + 'html content')
        },delay)
    }

    async.mapLimit(urls,1,
        function(url, callback){
            fetchUrl(url, callback)
        },
        function(err, result){
            console.log('final:')
            console.log(result)
            console.log(itemArr)
        })
}
// https://www.amazon.com/gp/search?ie=UTF8&qid=1509371076&lo=computers&rh=n%3A172282%2Cn%3A%21493964%2Cn%3A541966%2Cn%3A13896617011%2Cn%3A565098%2Cn%3A13896603011&page=4